From ff19a482909df1296041a61c1ddd48e9e8a084ff Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 11 Jul 2014 11:22:07 -0700 Subject: [PATCH] Finish plugin support This commit implements full support for plugins by answering the question of whether any target needed as a plugin or needed as a target dependency. This commit builds on the previous abstractions to enable parallel compilation wherever possible. --- src/cargo/ops/cargo_rustc/context.rs | 88 +++++++++++-- src/cargo/ops/cargo_rustc/fingerprint.rs | 4 +- src/cargo/ops/cargo_rustc/mod.rs | 136 +++++++++++--------- tests/test_cargo_compile.rs | 4 +- tests/test_cargo_cross_compile.rs | 156 ++++++++++++++++++++++- 5 files changed, 317 insertions(+), 71 deletions(-) diff --git a/src/cargo/ops/cargo_rustc/context.rs b/src/cargo/ops/cargo_rustc/context.rs index b8b0cee0f..56a5ab63b 100644 --- a/src/cargo/ops/cargo_rustc/context.rs +++ b/src/cargo/ops/cargo_rustc/context.rs @@ -1,28 +1,41 @@ use std::io::IoError; use std::io; use std::str; +use std::collections::{HashMap, HashSet}; -use core::{Package, PackageSet, Resolve, Target}; +use core::{Package, PackageId, PackageSet, Resolve, Target}; use util; use util::{CargoResult, ChainError, internal, Config}; +#[deriving(Show)] +pub enum PlatformRequirement { + Target, + Plugin, + PluginAndTarget, +} + pub struct Context<'a, 'b> { - pub deps_dir: Path, pub primary: bool, pub rustc_version: String, pub config: &'b mut Config<'b>, dest: Path, + host_dest: Path, + deps_dir: Path, + host_deps_dir: Path, host_dylib: (String, String), package_set: &'a PackageSet, resolve: &'a Resolve, target_dylib: (String, String), + requirements: HashMap<(&'a PackageId, &'a str), PlatformRequirement>, } impl<'a, 'b> Context<'a, 'b> { pub fn new(resolve: &'a Resolve, deps: &'a PackageSet, config: &'b mut Config<'b>, - dest: Path, deps_dir: Path) -> CargoResult> { + dest: Path, deps_dir: Path, + host_dest: Path, host_deps_dir: Path) + -> CargoResult> { let target_dylib = try!(Context::dylib_parts(config.target())); let host_dylib = if config.target().is_none() { target_dylib.clone() @@ -39,6 +52,9 @@ impl<'a, 'b> Context<'a, 'b> { config: config, target_dylib: target_dylib, host_dylib: host_dylib, + requirements: HashMap::new(), + host_dest: host_dest, + host_deps_dir: host_deps_dir, }) } @@ -72,17 +88,29 @@ impl<'a, 'b> Context<'a, 'b> { /// Prepare this context, ensuring that all filesystem directories are in /// place. - pub fn prepare(&self, pkg: &Package) -> CargoResult<()> { + pub fn prepare(&mut self, pkg: &'a Package) -> CargoResult<()> { debug!("creating target dir; path={}", self.dest.display()); try!(self.mk_target(&self.dest).chain_error(|| internal(format!("Couldn't create the target directory for {} at {}", pkg.get_name(), self.dest.display())))); + try!(self.mk_target(&self.host_dest).chain_error(|| + internal(format!("Couldn't create the host directory for {} at {}", + pkg.get_name(), self.dest.display())))); try!(self.mk_target(&self.deps_dir).chain_error(|| internal(format!("Couldn't create the directory for dependencies for {} at {}", pkg.get_name(), self.deps_dir.display())))); + try!(self.mk_target(&self.host_deps_dir).chain_error(|| + internal(format!("Couldn't create the directory for dependencies for {} at {}", + pkg.get_name(), self.deps_dir.display())))); + + let targets = pkg.get_targets().iter(); + for target in targets.filter(|t| t.get_profile().is_compile()) { + self.build_requirements(pkg, target, Target, &mut HashSet::new()); + } + Ok(()) } @@ -90,6 +118,30 @@ impl<'a, 'b> Context<'a, 'b> { io::fs::mkdir_recursive(target, io::UserRWX) } + fn build_requirements(&mut self, pkg: &'a Package, target: &'a Target, + req: PlatformRequirement, + visiting: &mut HashSet<&'a PackageId>) { + if !visiting.insert(pkg.get_package_id()) { return } + + let key = (pkg.get_package_id(), target.get_name()); + let req = if target.get_profile().is_plugin() {Plugin} else {req}; + self.requirements.insert_or_update_with(key, req, |_, v| { + *v = v.combine(req); + }); + + for &(pkg, dep) in self.dep_targets(pkg).iter() { + self.build_requirements(pkg, dep, req, visiting); + } + + visiting.remove(&pkg.get_package_id()); + } + + pub fn get_requirement(&self, pkg: &'a Package, + target: &'a Target) -> PlatformRequirement { + self.requirements.find(&(pkg.get_package_id(), target.get_name())) + .map(|a| *a).unwrap_or(Target) + } + /// Switch this context over to being the primary compilation unit, /// affecting the output of `dest()` and such. pub fn primary(&mut self) { @@ -97,8 +149,17 @@ impl<'a, 'b> Context<'a, 'b> { } /// Return the destination directory for output. - pub fn dest<'a>(&'a self) -> &'a Path { - if self.primary {&self.dest} else {&self.deps_dir} + pub fn dest<'a>(&'a self, plugin: bool) -> &'a Path { + if self.primary { + if plugin {&self.host_dest} else {&self.dest} + } else { + self.deps_dir(plugin) + } + } + + /// Return the destination directory for dependencies. + pub fn deps_dir<'a>(&'a self, plugin: bool) -> &'a Path { + if plugin {&self.host_deps_dir} else {&self.deps_dir} } /// Return the (prefix, suffix) pair for dynamic libraries. @@ -126,7 +187,7 @@ impl<'a, 'b> Context<'a, 'b> { /// For a package, return all targets which are registered as dependencies /// for that package. - pub fn dep_targets(&self, pkg: &Package) -> Vec { + pub fn dep_targets(&self, pkg: &Package) -> Vec<(&'a Package, &'a Target)> { let deps = match self.resolve.deps(pkg.get_package_id()) { None => return vec!(), Some(deps) => deps, @@ -139,9 +200,18 @@ impl<'a, 'b> Context<'a, 'b> { .filter_map(|pkg| { pkg.get_targets().iter().find(|&t| { t.is_lib() && t.get_profile().is_compile() - }) + }).map(|t| (pkg, t)) }) - .map(|t| t.clone()) .collect() } } + +impl PlatformRequirement { + fn combine(self, other: PlatformRequirement) -> PlatformRequirement { + match (self, other) { + (Target, Target) => Target, + (Plugin, Plugin) => Plugin, + _ => PluginAndTarget, + } + } +} diff --git a/src/cargo/ops/cargo_rustc/fingerprint.rs b/src/cargo/ops/cargo_rustc/fingerprint.rs index 7bee9f9b8..68b121d0c 100644 --- a/src/cargo/ops/cargo_rustc/fingerprint.rs +++ b/src/cargo/ops/cargo_rustc/fingerprint.rs @@ -18,8 +18,8 @@ use super::context::Context; /// compilation, returning the job as the second part of the tuple. pub fn prepare(cx: &mut Context, pkg: &Package, targets: &[&Target]) -> CargoResult<(Freshness, Job)> { - let fingerprint_loc = cx.dest().join(format!(".{}.fingerprint", - pkg.get_name())); + let fingerprint_loc = cx.dest(false).join(format!(".{}.fingerprint", + pkg.get_name())); let (is_fresh, fingerprint) = try!(is_fresh(pkg, &fingerprint_loc, cx, targets)); diff --git a/src/cargo/ops/cargo_rustc/mod.rs b/src/cargo/ops/cargo_rustc/mod.rs index 067acd0dd..e19daae8f 100644 --- a/src/cargo/ops/cargo_rustc/mod.rs +++ b/src/cargo/ops/cargo_rustc/mod.rs @@ -5,7 +5,7 @@ use util::{Config, Freshness}; use self::job::Job; use self::job_queue::JobQueue; -use self::context::Context; +use self::context::{Context, PlatformRequirement, Target, Plugin, PluginAndTarget}; mod context; mod fingerprint; @@ -32,7 +32,7 @@ fn uniq_target_dest<'a>(targets: &[&'a Target]) -> Option<&'a str> { curr.unwrap() } -pub fn compile_targets<'a>(env: &str, targets: &[&Target], pkg: &Package, +pub fn compile_targets<'a>(env: &str, targets: &[&'a Target], pkg: &'a Package, deps: &PackageSet, resolve: &'a Resolve, config: &'a mut Config<'a>) -> CargoResult<()> { @@ -42,13 +42,18 @@ pub fn compile_targets<'a>(env: &str, targets: &[&Target], pkg: &Package, debug!("compile_targets; targets={}; pkg={}; deps={}", targets, pkg, deps); + let host_dir = pkg.get_absolute_target_dir() + .join(uniq_target_dest(targets).unwrap_or("")); + let host_deps_dir = host_dir.join("deps"); + let target_dir = pkg.get_absolute_target_dir() .join(config.target().unwrap_or("")) .join(uniq_target_dest(targets).unwrap_or("")); let deps_target_dir = target_dir.join("deps"); let mut cx = try!(Context::new(resolve, deps, config, - target_dir, deps_target_dir)); + target_dir, deps_target_dir, + host_dir, host_deps_dir)); // First ensure that the destination directory exists try!(cx.prepare(pkg)); @@ -79,8 +84,10 @@ pub fn compile_targets<'a>(env: &str, targets: &[&Target], pkg: &Package, JobQueue::new(cx.config, jobs).execute() } -fn compile<'a>(targets: &[&Target], pkg: &'a Package, cx: &mut Context, - jobs: &mut Vec<(&'a Package, Freshness, Job)>) -> CargoResult<()> { +fn compile<'a, 'b>(targets: &[&'a Target], pkg: &'a Package, + cx: &mut Context<'a, 'b>, + jobs: &mut Vec<(&'a Package, Freshness, Job)>) + -> CargoResult<()> { debug!("compile_pkg; pkg={}; targets={}", pkg, targets); if targets.is_empty() { @@ -104,11 +111,12 @@ fn compile<'a>(targets: &[&Target], pkg: &'a Package, cx: &mut Context, // interdependencies. let (mut libs, mut bins) = (Vec::new(), Vec::new()); for &target in targets.iter() { - let job = rustc(pkg, target, cx); + let req = cx.get_requirement(pkg, target); + let jobs = rustc(pkg, target, cx, req); if target.is_lib() { - libs.push(job); + libs.push_all_move(jobs); } else { - bins.push(job); + bins.push_all_move(jobs); } } @@ -134,13 +142,15 @@ fn compile<'a>(targets: &[&Target], pkg: &'a Package, cx: &mut Context, fn compile_custom(pkg: &Package, cmd: &str, cx: &Context) -> Job { - // FIXME: this needs to be smarter about splitting + // TODO: this needs to be smarter about splitting let mut cmd = cmd.split(' '); + // TODO: this shouldn't explicitly pass `false` for dest/deps_dir, we may + // be building a C lib for a plugin let mut p = util::process(cmd.next().unwrap()) .cwd(pkg.get_root()) - .env("OUT_DIR", Some(cx.dest().as_str() + .env("OUT_DIR", Some(cx.dest(false).as_str() .expect("non-UTF8 dest path"))) - .env("DEPS_DIR", Some(cx.deps_dir.as_str() + .env("DEPS_DIR", Some(cx.deps_dir(false).as_str() .expect("non-UTF8 deps path"))) .env("TARGET", cx.config.target()); for arg in cmd { @@ -152,56 +162,73 @@ fn compile_custom(pkg: &Package, cmd: &str, }) } -fn rustc(package: &Package, target: &Target, cx: &mut Context) -> Job { +fn rustc(package: &Package, target: &Target, + cx: &mut Context, req: PlatformRequirement) -> Vec { let crate_types = target.rustc_crate_types(); let root = package.get_root(); - log!(5, "root={}; target={}; crate_types={}; dest={}; deps={}; verbose={}", - root.display(), target, crate_types, cx.dest().display(), - cx.deps_dir.display(), cx.primary); + log!(5, "root={}; target={}; crate_types={}; dest={}; deps={}; \ + verbose={}; req={}", + root.display(), target, crate_types, cx.dest(false).display(), + cx.deps_dir(false).display(), cx.primary, req); let primary = cx.primary; - let rustc = prepare_rustc(package, target, crate_types, cx); + let rustcs = prepare_rustc(package, target, crate_types, cx, req); - log!(5, "command={}", rustc); + log!(5, "commands={}", rustcs); let _ = cx.config.shell().verbose(|shell| { - shell.status("Running", rustc.to_string()) + for rustc in rustcs.iter() { + try!(shell.status("Running", rustc.to_string())); + } + Ok(()) }); - Job::new(proc() { - if primary { - log!(5, "executing primary"); - try!(rustc.exec().map_err(|err| human(err.to_string()))) - } else { - log!(5, "executing deps"); - try!(rustc.exec_with_output().and(Ok(())).map_err(|err| { - human(err.to_string()) - })) - } - Ok(Vec::new()) - }) + rustcs.move_iter().map(|rustc| { + Job::new(proc() { + if primary { + log!(5, "executing primary"); + try!(rustc.exec().map_err(|err| human(err.to_string()))) + } else { + log!(5, "executing deps"); + try!(rustc.exec_with_output().and(Ok(())).map_err(|err| { + human(err.to_string()) + })) + } + Ok(Vec::new()) + }) + }).collect() } fn prepare_rustc(package: &Package, target: &Target, crate_types: Vec<&str>, - cx: &Context) -> ProcessBuilder -{ + cx: &Context, req: PlatformRequirement) -> Vec { let root = package.get_root(); - let mut args = Vec::new(); - - build_base_args(&mut args, target, crate_types, cx); - build_deps_args(&mut args, package, cx); - - util::process("rustc") - .cwd(root.clone()) - .args(args.as_slice()) - .env("RUST_LOG", None) // rustc is way too noisy + let mut target_args = Vec::new(); + build_base_args(&mut target_args, target, crate_types.as_slice(), cx, false); + build_deps_args(&mut target_args, package, cx, false); + + let mut plugin_args = Vec::new(); + build_base_args(&mut plugin_args, target, crate_types.as_slice(), cx, true); + build_deps_args(&mut plugin_args, package, cx, true); + + let base = util::process("rustc").cwd(root.clone()); + + match req { + Target => vec![base.args(target_args.as_slice())], + Plugin => vec![base.args(plugin_args.as_slice())], + PluginAndTarget if cx.config.target().is_none() => + vec![base.args(target_args.as_slice())], + PluginAndTarget => + vec![base.clone().args(target_args.as_slice()), + base.args(plugin_args.as_slice())], + } } fn build_base_args(into: &mut Args, target: &Target, - crate_types: Vec<&str>, - cx: &Context) + crate_types: &[&str], + cx: &Context, + plugin: bool) { let metadata = target.get_metadata(); @@ -216,7 +243,6 @@ fn build_base_args(into: &mut Args, into.push(crate_type.to_string()); } - let out = cx.dest().clone(); let profile = target.get_profile(); if profile.get_opt_level() != 0 { @@ -244,16 +270,11 @@ fn build_base_args(into: &mut Args, None => {} } - if target.is_lib() { - into.push("--out-dir".to_string()); - into.push(out.display().to_string()); - } else { - into.push("-o".to_string()); - into.push(out.join(target.get_name()).display().to_string()); - } + into.push("--out-dir".to_string()); + into.push(cx.dest(plugin).display().to_string()); match cx.config.target() { - Some(target) if !profile.is_plugin() => { + Some(target) if !plugin => { into.push("--target".to_string()); into.push(target.to_string()); } @@ -261,17 +282,18 @@ fn build_base_args(into: &mut Args, } } -fn build_deps_args(dst: &mut Args, package: &Package, cx: &Context) { +fn build_deps_args(dst: &mut Args, package: &Package, cx: &Context, + plugin: bool) { dst.push("-L".to_string()); - dst.push(cx.dest().display().to_string()); + dst.push(cx.dest(plugin).display().to_string()); dst.push("-L".to_string()); - dst.push(cx.deps_dir.display().to_string()); + dst.push(cx.deps_dir(plugin).display().to_string()); - for target in cx.dep_targets(package).iter() { + for &(_, target) in cx.dep_targets(package).iter() { dst.push("--extern".to_string()); dst.push(format!("{}={}/{}", target.get_name(), - cx.deps_dir.display(), + cx.deps_dir(target.get_profile().is_plugin()).display(), cx.target_filename(target))); } } diff --git a/tests/test_cargo_compile.rs b/tests/test_cargo_compile.rs index 57ef09ceb..7c2d0affb 100644 --- a/tests/test_cargo_compile.rs +++ b/tests/test_cargo_compile.rs @@ -93,8 +93,8 @@ test!(cargo_compile_with_invalid_code { {filename}:1 invalid rust code! ^~~~~~~ Could not execute process \ -`rustc {filename} --crate-name foo --crate-type bin -o {} -L {} -L {}` (status=101)\n", - target.join("foo").display(), +`rustc {filename} --crate-name foo --crate-type bin --out-dir {} -L {} -L {}` (status=101)\n", + target.display(), target.display(), target.join("deps").display(), filename = format!("src{}foo.rs", path::SEP)).as_slice())); diff --git a/tests/test_cargo_cross_compile.rs b/tests/test_cargo_cross_compile.rs index 4bc4ddc60..9483e82d0 100644 --- a/tests/test_cargo_cross_compile.rs +++ b/tests/test_cargo_cross_compile.rs @@ -15,7 +15,7 @@ fn setup() { fn alternate() -> &'static str { match os::consts::SYSNAME { "linux" => "i686-unknown-linux-gnu", - "darwin" => "i686-apple-darwin", + "macos" => "i686-apple-darwin", _ => unreachable!(), } } @@ -75,4 +75,158 @@ test!(simple_deps { execs().with_status(0)); }) +test!(plugin_deps { + let foo = project("foo") + .file("Cargo.toml", r#" + [package] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies.bar] + path = "../bar" + + [dependencies.baz] + path = "../baz" + "#) + .file("src/main.rs", r#" + #![feature(phase)] + #[phase(plugin)] + extern crate bar; + extern crate baz; + fn main() { + assert_eq!(bar!(), baz::baz()); + } + "#); + let bar = project("bar") + .file("Cargo.toml", r#" + [package] + name = "bar" + version = "0.0.1" + authors = [] + [[lib]] + name = "bar" + plugin = true + "#) + .file("src/lib.rs", r#" + #![feature(plugin_registrar, quote)] + + extern crate rustc; + extern crate syntax; + + use rustc::plugin::Registry; + use syntax::ast::TokenTree; + use syntax::codemap::Span; + use syntax::ext::base::{ExtCtxt, MacExpr, MacResult}; + + #[plugin_registrar] + pub fn foo(reg: &mut Registry) { + reg.register_macro("bar", expand_bar); + } + + fn expand_bar(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) + -> Box { + MacExpr::new(quote_expr!(cx, 1i)) + } + "#); + let baz = project("baz") + .file("Cargo.toml", r#" + [package] + name = "baz" + version = "0.0.1" + authors = [] + "#) + .file("src/lib.rs", "pub fn baz() -> int { 1 }"); + bar.build(); + baz.build(); + + let target = alternate(); + assert_that(foo.cargo_process("cargo-build").arg("--target").arg(target), + execs().with_status(0)); + assert_that(&foo.target_bin(target, "main"), existing_file()); + + assert_that( + process(foo.target_bin(target, "main")), + execs().with_status(0)); +}) + +test!(plugin_to_the_max { + let foo = project("foo") + .file("Cargo.toml", r#" + [package] + name = "foo" + version = "0.0.1" + authors = [] + + [dependencies.bar] + path = "../bar" + + [dependencies.baz] + path = "../baz" + "#) + .file("src/main.rs", r#" + #![feature(phase)] + #[phase(plugin)] + extern crate bar; + extern crate baz; + fn main() { + assert_eq!(bar!(), baz::baz()); + } + "#); + let bar = project("bar") + .file("Cargo.toml", r#" + [package] + name = "bar" + version = "0.0.1" + authors = [] + + [[lib]] + name = "bar" + plugin = true + + [dependencies.baz] + path = "../baz" + "#) + .file("src/lib.rs", r#" + #![feature(plugin_registrar, quote)] + + extern crate rustc; + extern crate syntax; + extern crate baz; + + use rustc::plugin::Registry; + use syntax::ast::TokenTree; + use syntax::codemap::Span; + use syntax::ext::base::{ExtCtxt, MacExpr, MacResult}; + + #[plugin_registrar] + pub fn foo(reg: &mut Registry) { + reg.register_macro("bar", expand_bar); + } + + fn expand_bar(cx: &mut ExtCtxt, sp: Span, tts: &[TokenTree]) + -> Box { + MacExpr::new(quote_expr!(cx, baz::baz())) + } + "#); + let baz = project("baz") + .file("Cargo.toml", r#" + [package] + name = "baz" + version = "0.0.1" + authors = [] + "#) + .file("src/lib.rs", "pub fn baz() -> int { 1 }"); + bar.build(); + baz.build(); + + let target = alternate(); + assert_that(foo.cargo_process("cargo-build").arg("--target").arg(target), + execs().with_status(0)); + assert_that(&foo.target_bin(target, "main"), existing_file()); + + assert_that( + process(foo.target_bin(target, "main")), + execs().with_status(0)); +}) -- 2.30.2